/**********************************************************************/
/*
      Title: raw_to_importable.do
			Author: Robbie Dulin, Clotaire Boyer
			Created: 30 Oct 2019
    	Description: Saves raw data files in importable (.dta) format when not already in .dta
*/
/**********************************************************************/

clear
set more off

/*----------------------------------------------------*/
            /* Section 1: Import raw data */
/*----------------------------------------------------*/

cap log close
local prefix: display %tdCYND td(`c(current_date)')
log using "$log/`prefix'_raw_to_importable", replace

***** March 2018 SUSENAS ******
global raw_dbf                      "$dropboxbase/Susenas"
*** Import blok 41 data
import dbase using "$raw_dbf/Susenas 2018 Maret_full/Susenas Mar 2018 MKP/data/blok41_gab_prop11-33_diseminasi.dbf", clear
tempfile b41prop11_33
save `b41prop11_33'
import dbase using "$raw_dbf/Susenas 2018 Maret_full/Susenas Mar 2018 MKP/data/blok41_gab_prop34-94_diseminasi.dbf", clear
append using `b41prop11_33'
save "${importable}/blok41_gab_prop11-94_mar_18.dta" , replace

*** Import blok 42 data
import dbase using "$raw_dbf/Susenas 2018 Maret_full/Susenas Mar 2018 MKP/data/blok42.dbf", clear
save "$importable/blok42_mar_18.dta", replace

*** Import blok43 data
import dbase using "$raw_dbf/Susenas 2018 Maret_full/Susenas Mar 2018 MKP/data/blok43.dbf", clear
save "${importable}/blok43_mar_18.dta" , replace

*** Import household module data
cd "$raw_dbf/Susenas 2018 Maret_full/Susenas Mar 2018 KOR/Data Mikro Survei Sosial Ekonomi Nasional 2018 Maret (KOR)"
import dbase "kor18rt_diseminasi.dbf", clear
// import dbase "$raw_dbf/Susenas 2018 Maret_full/Susenas Mar 2018 KOR/Data Mikro Survei Sosial Ekonomi Nasional 2018 Maret (KOR)/kor18rt_diseminasi.dbf", clear
save "${importable}/kor18rt_diseminasi.dta", replace

*** Import individual module data
// import dbase "kor18ind_revisi_diseminasi.dbf", clear
import dbase "kor18ind_diseminasi.dbf", clear
// import dbase "$raw_dbf/Susenas 2018 Maret_full/Susenas Mar 2018 KOR/Data Mikro Survei Sosial Ekonomi Nasional 2018 Maret (KOR)/kor18ind_revisi_diseminasi.dbf", clear
save "${importable}/kor18ind_diseminasi.dta", replace


***** March 2019 SUSENAS *****

*** Ind
import dbase "$raw_dbf/SUSENAS Mar 2019/From Bu Vivi Jan 2020/KOR Individu/kor19ind_2_dds.dbf", clear
tempfile kor19ind2
save `kor19ind2'
import dbase "$raw_dbf/SUSENAS Mar 2019/From Bu Vivi Jan 2020/KOR Individu/kor19ind_1_dds.dbf", clear
merge 1:1 RENUM R401 using `kor19ind2', assert(3)
drop _m

save "$importable/kor19ind.dta", replace

*** RT
import dbase "$raw_dbf/SUSENAS Mar 2019/From Bu Vivi Jan 2020/kor19rt_dds.dbf", clear
save "$importable/kor19rt.dta", replace

*** Blok 41
import dbase "$raw_dbf/SUSENAS Mar 2019/From Bu Vivi Jan 2020/KP blok 41/blok41_gab_31_36_dds.dbf", clear
tempfile b41_31_36
save `b41_31_36'
import dbase "$raw_dbf/SUSENAS Mar 2019/From Bu Vivi Jan 2020/KP blok 41/blok41_gab_51_94_dds.dbf", clear
tempfile b41_51_94
save `b41_51_94'
import dbase "$raw_dbf/SUSENAS Mar 2019/From Bu Vivi Jan 2020/KP blok 41/blok41_gab_11_21_dds.dbf", clear
append using `b41_31_36'
append using `b41_51_94'
save "$importable/blok41_mar19.dta", replace

*** Blok 42
import dbase "$raw_dbf/SUSENAS Mar 2019/From Bu Vivi Jan 2020/KP blok 42/blok42_31_36_dds.dbf", clear
tempfile b42_31_36
save `b42_31_36'
import dbase "$raw_dbf/SUSENAS Mar 2019/From Bu Vivi Jan 2020/KP blok 42/blok42_51_94_dds.dbf", clear
tempfile b42_51_94
save `b42_51_94'
import dbase "$raw_dbf/SUSENAS Mar 2019/From Bu Vivi Jan 2020/KP blok 42/blok42_11_21_dds.dbf", clear
append using `b42_31_36'
append using `b42_51_94'
save "$importable/blok42_mar19.dta", replace

*** Blok 43
import dbase "$raw_dbf/SUSENAS Mar 2019/From Bu Vivi Jan 2020/blok43_dds.dbf", clear
save "$importable/blok43_mar19.dta", replace

*** Import PODES 2018 data
global raw_dbf                      "$dropboxbase/Podes 2018/Desa"
forvalues num = 1 / 4 {
  import dbase "$raw_dbf/PODES DESA/podes2018_desa_part`num'_revisi.dbf", clear
  save "$importable/podes2018_desa_part`num'_revisi.dta", replace
}

/**
Ran once with J-Pal data, not accessible to us since.
*** Import PODES 2019 data
import dbase "$jpal_data/Podes/Podes 2019/DATA/podes_2019_b1b8.dbf", clear

preserve
import dbase "$jpal_data/Podes/Podes 2019/DATA/podes_2019_b9b13.dbf", clear
tempfile podes19
qui save `podes19'
restore

merge 1:1 R101 R102 R103 R104 using `podes19', nogen assert(3)

save "$importable/podes_2019_b1b13.dta", replace
**/
cap log close
